package com.spider.common.task.service;

import com.alibaba.fastjson.JSONObject;
import com.spider.common.bean.AgentVo;
import com.spider.common.bean.Category;
import com.spider.common.bean.ganji.GjHouseDetailVo;
import com.spider.common.bean.ganji.GjHousePhoneVo;
import com.spider.common.bean.ganji.HouseVo;
import com.spider.common.utils.JedisUtils;
import com.spider.common.utils.MD5Util;
import com.spider.common.utils.ObjectUtils;
import com.spider.common.utils.StringUtils;
import com.spider.common.web.mapper.CategoryMapper;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.conf.XxlCrawlerConf;
import com.xuxueli.crawler.parser.PageParser;
import com.xuxueli.crawler.proxy.ProxyMaker;
import com.xuxueli.crawler.proxy.strategy.RoundProxyMaker;
import org.apache.http.client.utils.DateUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.*;

/**
 * @author marver
 * @Description 赶集抓取任务
 * @date 2018/12/17 9:35
 */
@Service
public class GanjiGrabTaskService {
    private static Logger logger = LoggerFactory.getLogger(GanjiGrabTaskService.class);
    //每次处理30条
    private final int pageSize = 30;
    @Autowired
    private CategoryMapper categoryMapper;

    /**
     * 1.获取首页分类
     * @param url
     * @param type
     */
    public void getHouse(String url, Integer type){

        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls(url)
                .setWhiteUrlRegexs(url)
                .setThreadCount(3)
                .setAllowSpread(false) //不允许扩散爬取
                .setPageParser(new PageParser<HouseVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, HouseVo pageVo) {
                        // 解析封装 PageVo 对象
                        String pageUrl = html.baseUri();
                        System.out.println(pageUrl + "：" + pageVo.toString());
                        addHouseCategory(pageVo, type);
                    }
                }).build();
        crawler.start(true);
    }

    /**
     * 添加爬取的分类
     * @param pageVo
     * @return
     */
    @Transactional
    public int addHouseCategory(HouseVo pageVo, Integer type){
        Category category = new Category();
        category.setCreateTime(new Date());
        category.setParentId(0L);
        category.setTitle(pageVo.getCategoryOne());
        category.setType(type);//1-58同城 2赶集
        int num = categoryMapper.insertSelective(category);

        List<Category> categories = new ArrayList<>();
        if(!CollectionUtils.isEmpty(pageVo.getCategoryTwo())){
            for (int i = 0; i < pageVo.getCategoryTwo().size(); i++) {
                Category category1 = new Category();
                category1.setUrl(pageVo.getUrl().get(i));
                category1.setType(type);
                category1.setTitle(pageVo.getCategoryTwo().get(i));
                category1.setParentId(category.getId());
                category1.setCreateTime(new Date());

                categories.add(category1);
            }
            categoryMapper.insertList(categories);
        }
        return num;
    }

    /**
     * 2、抓取赶集房屋详情页面
     * @param detailUrl
     */
    public List<String> getHouseDetail(String... detailUrl) {
        List<String> vos = new ArrayList<>();
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls(detailUrl)
                .setWhiteUrlRegexs(detailUrl)
                .setThreadCount(3)
                .setPageParser(new PageParser<GjHouseDetailVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, GjHouseDetailVo pageVo) {
                        // 解析封装 PageVo 对象
                        String pageUrl = html.baseUri();
                        //logger.debug(pageUrl + "：" + pageVo.toString());
                        if(StringUtils.isNotBlank(pageVo.getHtmlHref())){
                            vos.add(pageVo.getHtmlHref());
                        }
                    }
                }).build();
        logger.info("获取详情列表 start.....");
        crawler.start(true);
        logger.info("获取详情列表 end.....");

        return vos;
    }


    /**
     * 抓取电话信息
     * @param vos
     * @return
     */
    public List<GjHousePhoneVo> queryPhoneInfo(List<String> vos){
        try {
            Thread.sleep(5000);
            logger.info("第一次睡眠5秒");
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        List<GjHousePhoneVo> list = new ArrayList<>();
        //进行分页处理
        int num = vos.size()/pageSize;
        if(vos.size()%pageSize!=0){
            num +=1;
        }

        for (int i = 0; i < num; i++) {
            int nums = 0;
            for (int j = 0; j < pageSize && nums < vos.size(); j++) {
                nums = (i * pageSize) + j;
                Set<GjHousePhoneVo> voList = getHousePhone(vos.get(nums));
                list.addAll(voList);
                nums++;
            }
            try {
                Thread.sleep(3000);
                logger.info("睡眠5秒");
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        return list;
    }


    public Set<GjHousePhoneVo> queryPhoneInfos(List<String> vos){
        Set<GjHousePhoneVo> list = new HashSet<>();
       /* try {
            Thread.sleep(5000);
            logger.info("第一次睡眠5秒");
        } catch (InterruptedException e) {
            e.printStackTrace();
        }*/

        //进行分页处理
       /* String[] phoneUrls = new String[vos.size()];
        for (int i = 0; i < vos.size(); i++) {
            phoneUrls[i] = vos.get(i);
        }
        List<GjHousePhoneVo> voList = getHousePhone(phoneUrls);
        */

        //进行分页处理
        int num = vos.size()/pageSize;
        if(vos.size()%pageSize!=0){
            num +=1;
        }

        for (int i = 0; i < num; i++) {
            int nums = 0;
            String[] phoneUrls = new String[pageSize];
            for (int j = 0; j < pageSize && nums < vos.size(); j++) {
                nums = (i * pageSize) + j;
                //验证是否已爬取
                if(visitedDetailUrl(vos.get(nums))){
                    phoneUrls[j] = vos.get(nums);
                }
                nums++;
            }
            if(StringUtils.isBlank(phoneUrls[0])){
                return list;
            }
            Set<GjHousePhoneVo> voList = getHousePhone(phoneUrls);
            list.addAll(voList);
            /*try {
                logger.info("睡眠20秒");
                Thread.sleep(20000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }*/
            break;
        }
        return list;
    }

    /**
     * 3、获取电话号码
     * @param phoneUrl
     */
    public Set<GjHousePhoneVo> getHousePhone(String... phoneUrl){
        String[] userAgents = new String[]{XxlCrawlerConf.USER_AGENT_CHROME,XxlCrawlerConf.USER_AGENT_EDGE,
                XxlCrawlerConf.USER_AGENT_FIREFOX_45,XxlCrawlerConf.USER_AGENT_IE};
        Random rand = new Random();
        String userAgent = userAgents[rand.nextInt(4)];
        Set<GjHousePhoneVo> vos = new HashSet<>();
        //获取代理
        ProxyMaker proxyMaker = getProxy();

        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls(phoneUrl)
                //.setWhiteUrlRegexs("")
                .setThreadCount(3)
                .setPauseMillis(500)
                .setUserAgent(userAgent)
                .setProxyMaker(proxyMaker)
                .setAllowSpread(false) //不允许扩散爬取
                .setPageParser(new PageParser<GjHousePhoneVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, GjHousePhoneVo pageVo) {
                        logger.info("phone：" + pageVo.getPhone());
                        if(ObjectUtils.isChinaPhone(pageVo.getPhone())){
                            vos.add(pageVo);
                        }
                    }
                }).build();

        logger.info("获取房屋联系电话号码 start.....");
        crawler.start(true);
        logger.info("获取房屋联系电话号码 end.....");

        return vos;
    }

    /**
     * redis存取详情的url列表
     * @param url
     * @return
     */
    public List<String> redisHouseDetail(String url){
        List<String> urlList;
        //取出redis详情列表数据  存在则进行判断
        String nowDate = DateUtils.formatDate(new Date());
        /**
         * unVisitedUrl：待采集URL池
         * visitedUrl：已采集URL池
         */
        String unVisitedUrl = nowDate+":houseDetails:unVisitedUrl";
        urlList = JSONObject.parseArray(JedisUtils.get(unVisitedUrl),String.class);
        if(CollectionUtils.isEmpty(urlList)){
            urlList = getHouseDetail(url);
            JedisUtils.set(unVisitedUrl, JSONObject.toJSONString(urlList),0);
        }
        return urlList;
    }

    /**
     * 采集电话号码时执行
     * redis已采集URL列表收录
     * @param url
     * @return
     */
    public boolean visitedDetailUrl(String url){

        //取出redis详情列表数据  存在则进行判断
        String nowDate = DateUtils.formatDate(new Date());
        /**
         * unVisitedUrl：待采集URL池
         * visitedUrl：已采集URL
         */
        String unVisitedUrl = nowDate+":houseDetails:unVisitedUrl";
        String visitedUrl = nowDate+":houseDetails:visitedUrl:";

        //取出待采集的url 跟需要进行的url比较  不同则进行采集
        String visitedUrlData = JedisUtils.get(visitedUrl+MD5Util.returnPWD(url));
        if(!url.equals(visitedUrlData)){
            List<String> urlList = JSONObject.parseArray(JedisUtils.get(unVisitedUrl),String.class);
            for (int i = 0; i < urlList.size(); i++) {
                if(urlList.get(i).equals(url)){
                    urlList.remove(i);
                    break;
                }
            }
            String urlKey = MD5Util.returnPWD(url);
            JedisUtils.set(visitedUrl+urlKey, url, 0);
            JedisUtils.set(unVisitedUrl, JSONObject.toJSONString(urlList), 0);
            return true;
        }else {
            return false;
        }
    }

    //获取代理池
    public ProxyMaker getProxy(){
        ProxyMaker proxyMaker;
        try {
            List<AgentVo> proxyPool = JSONObject.parseArray(JedisUtils.get("agents"), AgentVo.class);
            if(proxyPool.size()>0){
                List<Proxy> proxyList = new ArrayList<>();
                proxyPool.forEach(p->{
                    Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(p.getIp(), p.getPort()));
                    proxyList.add(proxy);
                });
                proxyMaker = new RoundProxyMaker().addProxyList(proxyList);
            }else{
                proxyMaker = null;
            }
        } catch (Exception e) {
            proxyMaker = null;
        }
        return proxyMaker;
    }

    public static void main(String[] args) {
        GanjiGrabTaskService ganjiGrabTaskService = new GanjiGrabTaskService();
        ganjiGrabTaskService.getHouseDetail("https://sz.58.com/shangpucz/");
    }
}
